# Updated by Wei on 2014/07/06
# Using the original threshold to evaluate is WRONG.
# Use some cutoff instead please
exit(1)

# Updated by Wei on 2014/07/01
# To make it fit for the gov2, 150 HJ queries
# Updated by Wei on 2014/06/30
# To make it fit for the clueweb09B, 200 HJ queries
# and also want to revenge
# Updated by Wei on 2014/06/29
# When exploring the issues of cutoff, and I found that the water is too deep.
# so I decided to use back the original cutoff threshold for each query, and the evaluation will be more realistic. 
from __future__ import division
from operator import itemgetter, attrgetter
from struct import *
import gc
import math
import os
import random
import sys
import time
from sets import Set

print "Program Begins..."
if len(sys.argv) != 3:
    print "Illegal # of arguments"
    print "Usage: python programName.py allPostingPoppedFileName outputFileBaseName"
    exit(1)
else:
    pass

# FOR tail5K100K_GOV2
# for pangolin:
# inputFileName0 = "/data/obukai/workspace_USE_SINCE_20140217Night/web-search-engine-wei-2014-March/data/realFreqOfTermsForTail5KFrom100K"
# for dodo:
# inputFileName0 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/realFreqOfTermsForTail5KFrom100K"
# for moa:
# inputFileName0 = "/home/diaosi/workspace/web-search-engine-wei-2014-April/data/realFreqOfTermsForTail5KFrom100K"
# for vidaserver1:
# inputFileName0 = "/home/vgc/wei/workspace/NYU_IRTK/data/realFreqOfTermsForTail5KFrom100K_GOV2"
# FOR 200 HJ queries for clueweb09B
# for vidaserver1:
# inputFileName0 = "/home/vgc/wei/workspace/NYU_IRTK/data/realFreqOfTermsFor200HJQueries_Clueweb09B"
# FOR 150 HJ queries for gov2
inputFileName0 = "/home/vgc/wei/workspace/NYU_IRTK/data/realFreqOfTermsFor150HJQueries_GOV2"

# OR Semantics
# FOR tail5K100K_GOV2
# for pangolin:
# inputFileName1 = "/data/obukai/workspace_USE_SINCE_20140217Night/web-search-engine-wei-2014-March/data/rawResults_50%_TOP10_OR_20140126Night"
# for dodo:
# inputFileName1 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/rawResults_50%_TOP10_OR_20140126Night"
# for moa:
# inputFileName1 = "/home/diaosi/workspace/web-search-engine-wei-2014-April/data/rawResults_50%_TOP10_OR_20140126Night"
# for vidaserver1:
# inputFileName1 = "/home/vgc/wei/workspace/NYU_IRTK/data/rawResults_50%_TOP10_OR_20140126Night_GOV2"

# FOR 200 human judge queries_CLUEWEB09B
# for vidaserver1:
# inputFileName1 = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/clueweb09B_09_12_rawResults_20140630_TOP1000_OR"
# FOR 150 HJ queries for gov2
# TOP1000
# inputFileName1 = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/gov2_04_06_rawResults_20140701_TOP1000_OR"
# TOP100
inputFileName1 = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/gov2_04_06_rawResults_20140701_TOP100_OR"

# FOR tail5K100K_GOV2
# for pangolin:
# inputFileName2 = "/data/obukai/gov2ClearYourMindAndDoItAgain2014/partOfLexiconTermsWithTermIDForTail5KQueries"
# for dodo:
# inputFileName2 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/partOfLexiconTermsWithTermIDForTail5KQueries"
# for moa:
# inputFileName2 = "/home/diaosi/workspace/web-search-engine-wei-2014-April/data/partOfLexiconTermsWithTermIDForTail5KQueries"
# for vidaserver1:
# inputFileName2 = "/home/vgc/wei/workspace/NYU_IRTK/data/partOfLexiconTermsWithTermIDForTail5KQueries_GOV2"

# FOR 200 human judge queries_CLUEWEB09B
# for vidaserver1:
# inputFileName2 = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/clueweb09BQueryTermsFromHJQueries_09_12_20140624"
# FOR 150 HJ queries for gov2
# for vidaserver1:
inputFileName2 = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/gov2QueryTermsFromHJQueries_04_06_20140701"

# all posting being popped files:
# some options:
# for pangolin:
# inputFileName3 = "/data/obukai/workspace_USE_SINCE_20140217Night/web-search-engine-wei-2014-March/results/allPostingsBeingPopped20140321Morning_weight_0_WHOLE"
# for dodo:
# inputFileName3 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/allPostingsBeingPopped20140321Morning_weight_150_WHOLE"
# for moa:
# inputFileName3 = "/home/diaosi/workspace/web-search-engine-wei-2014-April/results/allPostingsBeingPopped20140402Morning_weight_0_SPECIAL_Pt_1"
# inputFileName3 = "/home/diaosi/workspace/web-search-engine-wei-2014-April/results/allPostingsBeingPopped20140321Morning_weight_0_WHOLE"
# for vidaserver1:
# allPostingPopped_GOV2_ourApproach_ptPowTo0_since201406280
# allPostingPopped_GOV2_ourApproach_ptPowToDot1_since201406280
# allPostingPopped_GOV2_ourApproach_ptPowToDot3_since201406280
# allPostingPopped_GOV2_ourApproach_ptPowToDot5_since201406280
# allPostingPopped_GOV2_ourApproach_ptPowToDot7_since201406280
# allPostingPopped_GOV2_ourApproach_ptPowToDot9_since201406280
# allPostingsBeingPopped20140321Morning_weight_0_around1Percent # for sanity check
inputFileName3 = sys.argv[1]

# This file is of NO USE any more. 
# Updated by Wei on 20140701 
# FOR tail5K100K_GOV2
# for dodo:
# WRONG Careful about the format inputFileName4 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/queryTermsWithTheirLengthsOfInvertedList"
# for pangolin
# WRONG Careful about the format inputFileName4 = "/data/obukai/workspace_USE_SINCE_20140217Night/web-search-engine-wei-2014-March/data/queryTermsWithTheirLengthsOfInvertedList"
# for moa:
# WRONG Careful about the format inputFileName4 = "/home/diaosi/workspace/web-search-engine-wei-2014-April/data/queryTermsWithTheirLengthsOfInvertedList"
# for vidaserver1:
# WRONG Careful about the format inputFileName4 = "/home/vgc/wei/workspace/NYU_IRTK/data/queryTermsWithTheirLengthsOfInvertedList_GOV2"

# FOR 200 human judge queries_CLUEWEB09B
# inputFileName4 = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/clueweb09BQueryTermsFromHJQueries_09_12_20140624"
# FOR 150 HJ queries for gov2
# for vidaserver1:
# inputFileName4 = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/gov2QueryTermsFromHJQueries_04_06_20140701"


# "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/queryProcessingCostAnalysis_weight_0_ptPowToDot9_OR_20140629_"
outputBaseFileName = sys.argv[2]

def getQueryIDAndResultSetGivenResultFileForGoldStandardResultFile(inputFileName1,NUM_OF_TOP_RESULTS_NEEDED_TO_EVALUATE,termANDTermIDDict,qIDWithDocumentResultsDict,postingWithDocumentResultDict,qIDWithThresholdDict,qIDWithQueryTermsDict,queryTermFreqDictFromCurrent):
    print "getQueryIDAndResultSetGivenResultFileForGoldStandardResultFile() function begins..."
    #tempQueryTermDict = {}
    #tempOutputFileName = "/data/obukai/gov2ClearYourMindAndDoItAgain2014/partOfLexiconTermsWithTermIDForTail5KQueries"
    #tempOutputFileHandler = open(tempOutputFileName,"w")
    
    tempCounter = 0
    tempCounter2 = 0
    numOfQueriesHavingQID = 0
    numOfQueriesHavingSearchContent = 0
    numOfQueriesHavingSearchResults = 0
    
    inputFileHandler1 = open(inputFileName1,"r")
    
    
    currentLine = inputFileHandler1.readline()
    nextLine = ""
    currentQID = ""
    
    numOfResultsForTheAllQueries = 0
    
    while currentLine:
        # sampling parsing line:
        # qid: 701
        if currentLine.startswith("qid:"):
            currentQID = currentLine.strip().split(":")[1].strip()
            print "currentQID:",currentQID
            numOfQueriesHavingQID += 1
        
        # sample parsing line:
        # Search: u s  oil industry history
        if currentLine.startswith("Search:"):
            elementList = currentLine.strip().split(" ")
            
            currentSearchContent = ""
            
            queryTermList = []
            for element in elementList[1:]:
                if element.strip() != "":
                    queryTermList.append( element.strip() )        
            currentQueryLength = len(queryTermList)
            print "currentQueryLength:",currentQueryLength
    
            for element in queryTermList:
                currentSearchContent += element + " "
            
            print currentSearchContent
            numOfQueriesHavingSearchContent += 1
            
            
            currentQueryTermIndexDict = {}
            nextLine = inputFileHandler1.readline()
            # print nextLine.strip()
            parsedFlag = True
            
            if nextLine.strip().endswith("is NOT in the lexicon."):
                parsedFlag = False
            
            if nextLine.strip() != "" and parsedFlag:
                
                # skip the following sampling line: 
                # [2013-05-24 14:20:44] 38871 terms with their lists length have been loaded.
                # nextLine = inputFileHandler1.readline()
                
                nextLineElements = nextLine.strip().split(" ")
                # print "nextLineElements:",nextLineElements
                
                # sample parsing line: 
                # oil:0 industry:1 history:2 u:3 s:4
                for element in nextLineElements:
                    if element.split(":")[0] not in currentQueryTermIndexDict:
                        currentQueryTermIndexDict[ int(element.split(":")[1]) ] = element.split(":")[0]
                
                print "currentQueryTermIndexDict:",currentQueryTermIndexDict
                # print "len(currentQueryTermIndexDict):",len(currentQueryTermIndexDict)
                if currentQID not in qIDWithQueryTermsDict:
                    qIDWithQueryTermsDict[currentQID] = []
                '''
                # test version1
                upperBound = 0
                if len(currentQueryTermIndexDict) >= 10:
                    upperBound = 10
                else:
                    upperBound = len(currentQueryTermIndexDict)
                
                for i in range(0,upperBound):
                    currentTerm = currentQueryTermIndexDict[i]
                    qIDWithQueryTermsDict[currentQID].append(currentTerm)
                '''
                # test version2
                for i in range(0,len(currentQueryTermIndexDict)):
                    currentTerm = currentQueryTermIndexDict[i]
                    qIDWithQueryTermsDict[currentQID].append(currentTerm)
                    
                    if currentTerm not in queryTermFreqDictFromCurrent:
                        queryTermFreqDictFromCurrent[currentTerm] = 1
                    else:
                        queryTermFreqDictFromCurrent[currentTerm] += 1

                # temp section
                #for queryTermIndex in currentQueryTermIndexDict:
                #    currentQueryTerm = currentQueryTermIndexDict[queryTermIndex]
                #    if currentQueryTerm not in tempQueryTermDict:
                #        tempQueryTermDict[currentQueryTerm] = 1
                #        tempOutputFileHandler.write( currentQueryTerm + "\n")
                
                
                # Now, it is time to read the threshold line
                currentLine = inputFileHandler1.readline()
                if len(currentLine.strip().split(":")) == 2:
                    name = currentLine.strip().split(":")[0]
                    value = currentLine.strip().split(":")[1]
                    print name,"::",value
                else:
                    print "format problem"
                    exit(1)
                    
                # Now, it is time to read computation method line
                currentLine = inputFileHandler1.readline()
                if len(currentLine.strip().split(":")) == 2:
                    name = currentLine.strip().split(":")[0]
                    value = currentLine.strip().split(":")[1]
                    print name,"::",value
                else:
                    print "format problem"
                    exit(1)
                
                # skip one line
                # currentLine = inputFileHandler1.readline()
                
                # Now, it is time to read the actual training example line
                currentLine = inputFileHandler1.readline()
                numOfResultsForTheCurrentQuery = 0
                
                # for IMPORTANT DEBUG ONLY
                # print "len( currentLine.strip().split(" ") ):",len( currentLine.strip().split(" ") )
                # print "currentLine:",currentLine
                # exit(1)
                
                lineElements = currentLine.strip().split(" ")
                # for gov2
                # while len( lineElements ) == 25:
                # for clueweb09B
                while len( lineElements ) == 34:
                    theRank = int(lineElements[0])
                    # for gov2
                    # theDocID = lineElements[-3]
                    # theScore = float(lineElements[-4])
                    # for clueweb09B
                    theDocID = lineElements[-2]
                    theScore = float(lineElements[-3])                    
                    documentResultKey = currentQID + "_" + theDocID
                    
                    if currentQID not in qIDWithDocumentResultsDict:
                        qIDWithDocumentResultsDict[currentQID] = {}
                    
                    if theRank <= NUM_OF_TOP_RESULTS_NEEDED_TO_EVALUATE:
                        if documentResultKey not in qIDWithDocumentResultsDict:
                            qIDWithDocumentResultsDict[currentQID][documentResultKey] = 0.0
                            qIDWithThresholdDict[currentQID] = theScore; 
                        else:
                            print "duplicated document result."
                            print "documentResultKey:",documentResultKey
                            exit(1)
                        
                        upperBound = 0
                        if len(currentQueryTermIndexDict) >= 10:
                            upperBound = 10
                        else:
                            upperBound = len(currentQueryTermIndexDict)
                        
                        for i in range(0,upperBound):
                            currentTerm = currentQueryTermIndexDict[i]
                            if currentTerm not in termANDTermIDDict:
                                print currentTerm,"is NOT in the dict."
                                exit(1)
                            else:
                                currentTermID = termANDTermIDDict[currentTerm]
                            # for gov2
                            # currentTermScore = float( lineElements[11 + i] )
                            # for clueweb09B
                            currentTermScore = float( lineElements[1 + i] )
                            if currentTermScore != 0.0:
                                postingKey = currentTermID + "_" + theDocID
                                if postingKey not in postingWithDocumentResultDict:
                                    postingWithDocumentResultDict[postingKey] = []
                                
                                postingWithDocumentResultDict[postingKey].append(documentResultKey)    
                                
                                tempCounter += 1
                        tempCounter2 += len(currentQueryTermIndexDict)
                    else:
                        pass
                    


                    numOfResultsForTheCurrentQuery += 1
                    
                    currentLine = inputFileHandler1.readline()
                    lineElements = currentLine.strip().split(" ")
                   
                numOfResultsForTheAllQueries += numOfResultsForTheCurrentQuery
                
                # sampling method 1: among all the top10 documents, select those documents, open those documents and extract the corresponding postings.
                # This method make sure that once the document have been selected, ALL the coresponding postings will also be selected.
                # This sampling method has been developed on 2013/01/09
                
                print "numOfResultsForTheCurrentQuery:",numOfResultsForTheCurrentQuery
                print "tempCounter:",tempCounter
                print "tempCounter2:",tempCounter2
                print
            
        currentLine = inputFileHandler1.readline()
        
    print "numOfQueriesHavingQID:",numOfQueriesHavingQID
    print "numOfQueriesHavingSearchContent:",numOfQueriesHavingSearchContent
    print "numOfQueriesHavingSearchResults:",numOfQueriesHavingSearchResults
    print "getQueryIDAndResultSetGivenResultFileForGoldStandardResultFile() function ends."

# key: term
# value: freq

queryTermFreqDictFromCurrent = {}
qIDWithQueryTermsDict = {}
qIDWithThresholdDict = {}
qIDWithDocumentResultsDict = {}
postingWithDocumentResultDict = {}


termANDTermIDDict = {}
queryTermWithTheirLengthOfInvertedListDict = {}
inputFileHandler = open(inputFileName2,"r")
currentLine = inputFileHandler.readline()
currentLineNum = 0
while currentLine:
    if currentLineNum % 1000000 == 0:
        print currentLineNum,"lines loaded."
    currentLineElements = currentLine.strip().split(" ")
    currentTermID = currentLineElements[0]
    currentTerm = currentLineElements[1]
    currentTermListLength = int(currentLineElements[2])
    queryTermWithTheirLengthOfInvertedListDict[currentTerm] = currentTermListLength
    termANDTermIDDict[currentTerm] = currentTermID
    currentLine = inputFileHandler.readline()
    currentLineNum += 1
print "len(queryTermWithTheirLengthOfInvertedListDict):",len(queryTermWithTheirLengthOfInvertedListDict)
# print "queryTermWithTheirLengthOfInvertedListDict['you']:",queryTermWithTheirLengthOfInvertedListDict['you']
print "len(termANDTermIDDict): ",len(termANDTermIDDict)
inputFileHandler.close()

getQueryIDAndResultSetGivenResultFileForGoldStandardResultFile(inputFileName1,10,termANDTermIDDict,qIDWithDocumentResultsDict,postingWithDocumentResultDict,qIDWithThresholdDict,qIDWithQueryTermsDict,queryTermFreqDictFromCurrent)

print "OVERALL:"
documentResultCounter = 0
for currentQID in qIDWithDocumentResultsDict:
    documentResultCounter += len( qIDWithDocumentResultsDict[currentQID] )  
print "# of top10 document results:",documentResultCounter
# for clueweb09B:
# print "qIDWithDocumentResultsDict['2']:",qIDWithDocumentResultsDict['2']
# for gov2:
print "qIDWithDocumentResultsDict['701']:",qIDWithDocumentResultsDict['701']
print "qIDWithDocumentResultsDict['702']:",qIDWithDocumentResultsDict['702']

postingCounter = 0
for currentPostingKey in postingWithDocumentResultDict:
    postingCounter += len(postingWithDocumentResultDict[currentPostingKey])
print "# of top10 postings:",postingCounter

# for gov2
#print "qIDWithThresholdDict['95001'] threshold:",qIDWithThresholdDict['95001']
#print "qIDWithThresholdDict['95003'] threshold:",qIDWithThresholdDict['95003']
# for clueweb09B
# print "qIDWithThresholdDict['1'] threshold:",qIDWithThresholdDict['1']
# print "qIDWithThresholdDict['2'] threshold:",qIDWithThresholdDict['2']
# for gov2 HJ queries
print "qIDWithThresholdDict['701'] threshold:",qIDWithThresholdDict['701']
print "qIDWithThresholdDict['702'] threshold:",qIDWithThresholdDict['702']

# print "queryTermFreqDictFromCurrent['workers']:",queryTermFreqDictFromCurrent['workers']

#################################################################################################################
OUTPUT_FILE_INDEX = 0
# For gov2:
TOTAL_NUM_OF_POSTINGS = 6451948010
numOfPostingNeededToBePoppedAtDebugPercentage = 5000000
# For clueweb09B:
# TOTAL_NUM_OF_POSTINGS = 17075485964
# numOfPostingNeededToBePoppedAtDebugPercentage = 13660388

numOfPostingNeededToBePoppedAt1Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.01)
numOfPostingNeededToBePoppedAt2Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.02)
numOfPostingNeededToBePoppedAt3Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.03)
numOfPostingNeededToBePoppedAt4Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.04)
numOfPostingNeededToBePoppedAt5Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.05)
numOfPostingNeededToBePoppedAt6Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.06)
numOfPostingNeededToBePoppedAt7Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.07)
numOfPostingNeededToBePoppedAt8Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.08)
numOfPostingNeededToBePoppedAt9Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.09)
numOfPostingNeededToBePoppedAt10Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.1)
numOfPostingNeededToBePoppedAt15Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.15)
numOfPostingNeededToBePoppedAt20Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.2)
numOfPostingNeededToBePoppedAt30Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.3)
numOfPostingNeededToBePoppedAt40Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.4)
numOfPostingNeededToBePoppedAt50Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.5)
numOfPostingNeededToBePoppedAt60Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.6)
numOfPostingNeededToBePoppedAt70Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.7)
numOfPostingNeededToBePoppedAt80Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.8)
numOfPostingNeededToBePoppedAt90Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.9)
print "numOfPostingNeededToBePoppedAtDebugPercentage:",numOfPostingNeededToBePoppedAtDebugPercentage
print "numOfPostingNeededToBePoppedAt1Percentage:",numOfPostingNeededToBePoppedAt1Percentage
print "numOfPostingNeededToBePoppedAt2Percentage:",numOfPostingNeededToBePoppedAt2Percentage
print "numOfPostingNeededToBePoppedAt3Percentage:",numOfPostingNeededToBePoppedAt3Percentage
print "numOfPostingNeededToBePoppedAt4Percentage:",numOfPostingNeededToBePoppedAt4Percentage
print "numOfPostingNeededToBePoppedAt5Percentage:",numOfPostingNeededToBePoppedAt5Percentage
print "numOfPostingNeededToBePoppedAt6Percentage:",numOfPostingNeededToBePoppedAt6Percentage
print "numOfPostingNeededToBePoppedAt7Percentage:",numOfPostingNeededToBePoppedAt7Percentage
print "numOfPostingNeededToBePoppedAt8Percentage:",numOfPostingNeededToBePoppedAt8Percentage
print "numOfPostingNeededToBePoppedAt9Percentage:",numOfPostingNeededToBePoppedAt9Percentage
print "numOfPostingNeededToBePoppedAt10Percentage:",numOfPostingNeededToBePoppedAt10Percentage
print "numOfPostingNeededToBePoppedAt15Percentage:",numOfPostingNeededToBePoppedAt15Percentage
print "numOfPostingNeededToBePoppedAt20Percentage:",numOfPostingNeededToBePoppedAt20Percentage
print "numOfPostingNeededToBePoppedAt30Percentage:",numOfPostingNeededToBePoppedAt30Percentage
print "numOfPostingNeededToBePoppedAt40Percentage:",numOfPostingNeededToBePoppedAt40Percentage
print "numOfPostingNeededToBePoppedAt50Percentage:",numOfPostingNeededToBePoppedAt50Percentage
print "numOfPostingNeededToBePoppedAt60Percentage:",numOfPostingNeededToBePoppedAt60Percentage
print "numOfPostingNeededToBePoppedAt70Percentage:",numOfPostingNeededToBePoppedAt70Percentage
print "numOfPostingNeededToBePoppedAt80Percentage:",numOfPostingNeededToBePoppedAt80Percentage
print "numOfPostingNeededToBePoppedAt90Percentage:",numOfPostingNeededToBePoppedAt90Percentage
numOfTOP10PostingsRetained = 0
numOfTop10DocumentResultsRetained = 0
numOfBytes = 0
numOfPostingPopped = 0
numOfPostingBeInTOP10 = 0

inputFileHandler = open(inputFileName3,"rb")

print "inputFileName0:",inputFileName0
print "inputFileName1:",inputFileName1
print "inputFileName2:",inputFileName2
# print "inputFileName3:",inputFileName3
print "--->posting file to evaluate:",inputFileName3
statinfo = os.stat(inputFileName3)


fileSize = statinfo.st_size
print "file size:",fileSize
numOfResultsReturnedCountedUpTo10 = 0

# value:
# termID
# key:
# # of posting in the list length
termIDAndListLengthDict = {}

while numOfBytes < fileSize:
    # current version
    # each time, read the info of 1M postings
    byteStringBuffer = inputFileHandler.read( 1000000 * 16)
    byteStringBufferIndexPosition = 0
    for i in range(0,1000000):
        byteString = byteStringBuffer[byteStringBufferIndexPosition:byteStringBufferIndexPosition+16]
        byteStringBufferIndexPosition += 16
        (termID,docID,currentProbability,impactScore) = unpack( "2I2f", byteString)
        postingKey = str(termID) + "_" + str(docID)
        
        if str(termID) not in termIDAndListLengthDict:
            termIDAndListLengthDict[str(termID)] = 1
        else:
            termIDAndListLengthDict[str(termID)] += 1
        
        if postingKey in postingWithDocumentResultDict:
            numOfTOP10PostingsRetained += len(postingWithDocumentResultDict[postingKey])
            
        # debug
        # print "postingKey:",postingKey
        # print termID,docID,currentProbability,impactScore
        if postingKey in postingWithDocumentResultDict:
            # interesting issue solved
            # print "touched."
            for currentDocumentResultKey in postingWithDocumentResultDict[postingKey]:
                currentQID = currentDocumentResultKey.strip().split("_")[0]
                qIDWithDocumentResultsDict[currentQID][currentDocumentResultKey] += impactScore
                # tempCounter += 1

        if numOfPostingPopped == numOfPostingNeededToBePoppedAtDebugPercentage or \
        numOfPostingPopped == numOfPostingNeededToBePoppedAt1Percentage or \
        numOfPostingPopped == numOfPostingNeededToBePoppedAt2Percentage or \
        numOfPostingPopped == numOfPostingNeededToBePoppedAt3Percentage or \
        numOfPostingPopped == numOfPostingNeededToBePoppedAt4Percentage or \
        numOfPostingPopped == numOfPostingNeededToBePoppedAt5Percentage or \
        numOfPostingPopped == numOfPostingNeededToBePoppedAt6Percentage or \
        numOfPostingPopped == numOfPostingNeededToBePoppedAt7Percentage or \
        numOfPostingPopped == numOfPostingNeededToBePoppedAt8Percentage or \
        numOfPostingPopped == numOfPostingNeededToBePoppedAt9Percentage or \
        numOfPostingPopped == numOfPostingNeededToBePoppedAt10Percentage or \
        numOfPostingPopped == numOfPostingNeededToBePoppedAt15Percentage or \
        numOfPostingPopped == numOfPostingNeededToBePoppedAt20Percentage or \
        numOfPostingPopped == numOfPostingNeededToBePoppedAt30Percentage or \
        numOfPostingPopped == numOfPostingNeededToBePoppedAt40Percentage or \
        numOfPostingPopped == numOfPostingNeededToBePoppedAt50Percentage or \
        numOfPostingPopped == numOfPostingNeededToBePoppedAt60Percentage or \
        numOfPostingPopped == numOfPostingNeededToBePoppedAt70Percentage or \
        numOfPostingPopped == numOfPostingNeededToBePoppedAt80Percentage or \
        numOfPostingPopped == numOfPostingNeededToBePoppedAt90Percentage:
            # key: currentQID
            # value: # of document results maintained
            searchSystemDocumentResultRetainedFromFirstTierDict = {}
            numOfTOP10DocumentResultRetained = 0
            
            for currentQID in qIDWithDocumentResultsDict:
                for currentDocumentResultKey in qIDWithDocumentResultsDict[currentQID]:
                    if qIDWithDocumentResultsDict[currentQID][currentDocumentResultKey] >= qIDWithThresholdDict[currentQID]:
                        numOfTOP10DocumentResultRetained += 1
                        # debug
                        print "----->",str(currentQID),str(currentDocumentResultKey),str(qIDWithDocumentResultsDict[currentQID][currentDocumentResultKey]),str(qIDWithThresholdDict[currentQID]),"TOP10DocumentCandidatesRetained"
                        
                        
                        if currentQID not in searchSystemDocumentResultRetainedFromFirstTierDict:
                            searchSystemDocumentResultRetainedFromFirstTierDict[currentQID] = 1
                        else:
                            searchSystemDocumentResultRetainedFromFirstTierDict[currentQID] += 1

            print "numOfTOP10DocumentResultRetained:",numOfTOP10DocumentResultRetained
            print "numOfPostingPopped:",numOfPostingPopped
            print "numOfTOP10PostingsRetained:",numOfTOP10PostingsRetained
            print "len(termIDAndListLengthDict):",len(termIDAndListLengthDict)
            
            outputFileName = outputBaseFileName + str(OUTPUT_FILE_INDEX)
            outputFileHandler = open(outputFileName,"w")
            OUTPUT_FILE_INDEX += 1
            
            numOfDocumentResultsReturnedInTotalForThisQID = 0
            numOfDocumentResultsReturnedCurrentlyForThisQID = 0
            totalEvaluationCostFromPrunedIndex = 0
            totalEvaluationCostFromFullIndex = 0
            for currentQID in qIDWithDocumentResultsDict:
                currentQueryQPCFromPrunedIndex = 0
                currentQueryQPCFromFullIndex = 0
                # the computation of the following:
                # numOfDocumentResultsReturnedInTotalForThisQID
                # numOfDocumentResultsReturnedCurrentlyForThisQID
                numOfDocumentResultsReturnedInTotalForThisQID = len( qIDWithDocumentResultsDict[currentQID] )
                if currentQID in searchSystemDocumentResultRetainedFromFirstTierDict:
                    numOfDocumentResultsReturnedCurrentlyForThisQID = searchSystemDocumentResultRetainedFromFirstTierDict[currentQID]
                else:
                    numOfDocumentResultsReturnedCurrentlyForThisQID = 0
                
                for currentQueryTerm in qIDWithQueryTermsDict[currentQID]: # loop through each query term in the query
                    if currentQueryTerm in termANDTermIDDict:
                        currentQueryTermID = termANDTermIDDict[currentQueryTerm]
                        if currentQueryTermID in termIDAndListLengthDict:
                            currentQueryQPCFromPrunedIndex += termIDAndListLengthDict[currentQueryTermID]
                        if currentQueryTerm in queryTermWithTheirLengthOfInvertedListDict:
                            currentQueryQPCFromFullIndex += queryTermWithTheirLengthOfInvertedListDict[currentQueryTerm]
                
                # accumulate each query to the total
                totalEvaluationCostFromPrunedIndex += currentQueryQPCFromPrunedIndex
                totalEvaluationCostFromFullIndex += currentQueryQPCFromFullIndex
                outputLine = str(currentQID) + " " + str( numOfDocumentResultsReturnedCurrentlyForThisQID ) + " " + str( numOfDocumentResultsReturnedInTotalForThisQID ) + " " + str(currentQueryQPCFromPrunedIndex) + " " + str(currentQueryQPCFromFullIndex) + "\n" 
                outputFileHandler.write(outputLine)
                # debug
                #print outputLine.strip()
                #print totalEvaluationCostFromPrunedIndex,totalEvaluationCostFromFullIndex
                #print 
            print "totalEvaluationCostFromPrunedIndex:",totalEvaluationCostFromPrunedIndex
            print "totalEvaluationCostFromFullIndex:",totalEvaluationCostFromFullIndex
            print "outputFileName:",outputFileName
            print
            outputFileHandler.close()
        
        numOfBytes += 16
        numOfPostingPopped += 1
        if numOfPostingPopped % 1000000 == 0:
            print str(numOfPostingPopped),"postings have been examined."

inputFileHandler.close()

print "Note: This process will NEVER finish and will have an error"
print "OVERALL:"
print "inputFileName0:",inputFileName0
print "inputFileName1:",inputFileName1
print "inputFileName2:",inputFileName2
print "inputFileName3:",inputFileName3
exit(1)